package com.jml.mapreduce.多队列提交任务;

import com.jml.mapreduce.计数器数据清洗.ETLMapper;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;


public class ETLDriver {
    /**
     * VM options:  -DHADOOP_USER_NAME=root
     * ## 这个是main方法的两个参数，就是HDFS里面的读取文件路径，和输出的路径
     * Program arguments: /1.txt /output
     */
    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        Configuration configuration = new Configuration();

        //往yarn上提交需要添加得到配置。在Windows上向集群提交任务
        configuration.set("fs.defaultFS", "hdfs://hadoop101:8020");
        configuration.set("mapreduce.framework.name","yarn");
        configuration.set("mapreduce.app-submission.cross-platform","true");
        configuration.set("yarn.resourcemanager.hostname","hadoop102");

        /**
         * 第二个参数是队列的名字
         */
        configuration.set("mapred.job.queue.name", "hive");

        Job job = Job.getInstance(configuration);

        //job.setJarByClass(ETLDriver.class);
        job.setJar("自己打的jar包的具体文件路径");

        job.setMapperClass(ETLMapper.class);
        job.setNumReduceTasks(0);

        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(NullWritable.class);

        FileInputFormat.setInputPaths(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));

        boolean b = job.waitForCompletion(true);
        System.exit(b ? 0 : 1);
    }
}
